In [1]:
from IPython.display import HTML
HTML('''<script>
code_show=true; 
function code_toggle() {
 if (code_show){
 $('div.input').hide();
 } else {
 $('div.input').show();
 }
 code_show = !code_show
} 
$( document ).ready(code_toggle);
</script>
The raw code for this IPython notebook is by default hidden for easier reading.
To toggle on/off the raw code, click <a href="javascript:code_toggle()">here</a>.''')
Out[1]:
The raw code for this IPython notebook is by default hidden for easier reading. To toggle on/off the raw code, click here.
In [2]:
#hide warnings
HTML('''<script>
code_show_err=true; 
function code_toggle_err() {
 if (code_show_err){
 $('div.output_stderr').hide();
 } else {
 $('div.output_stderr').show();
 }
 code_show_err = !code_show_err
} 
$( document ).ready(code_toggle_err);
</script>
To toggle on/off output errors, click <a href="javascript:code_toggle_err()">here</a>.''')
Out[2]:
To toggle on/off output errors, click here.
In [3]:
print('Testing LexA lines Downstream to RC_SS33917')
print('Project A83, Daniel Bushey')
Testing LexA lines Downstream to RC_SS33917
Project A83, Daniel Bushey
In [4]:
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
In [5]:
import pandas as pd
import os
import sys
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import ccModules as cc
import ccModules2 as cc2

from IPython.html.widgets import interact
import pandas as pd
from IPython.display import IFrame
from bokeh.palettes import Spectral4
from bokeh.plotting import figure, output_file, show
/media/daniel/Windows1/Users/dnabu/Desktop/ResearchYogaWindows/DataJ/Programming/Python/Modules/tifffile.py:265: UserWarning: failed to import the optional _tifffile C extension module.
Loading of some compressed images will be very slow.
Tifffile.c can be obtained at http://www.lfd.uci.edu/~gohlke/
  "failed to import the optional _tifffile C extension module.\n"
/home/daniel/anaconda3/lib/python3.6/site-packages/IPython/html.py:14: ShimWarning: The `IPython.html` package has been deprecated since IPython 4.0. You should import from `notebook` instead. `IPython.html.widgets` has moved to `ipywidgets`.
  "`IPython.html.widgets` has moved to `ipywidgets`.", ShimWarning)
In [6]:
#load project  specific parameters
from A83_init20190411 import *
path1 /media/daniel/Seagate5TB/A83/A83_data4
In [7]:
#load data
#load pandas data frame
exceldata = pd.read_hdf(os.path.join(saveFig, 'Compiled_data.hdf5'), 'data')
#print(exceldata.shape)
#exceldata = exceldata[exceldata['Grade'] == 1]
In [8]:
print('Checking how many sessions included in the loaded data set:')
##filter for only responders
exceldata = exceldata[exceldata['Responds']==1]
print(len(exceldata))
Checking how many sessions included in the loaded data set:
6
In [9]:
print('Stim Protocols')
exceldata['Stim Protocol'].unique()
Stim Protocols
Out[9]:
array(['stim30s06V', 'stimDur300at15'], dtype=object)
In [10]:
#combine stim protocols into one data frame
shortlist1 = exceldata[exceldata['Stim Protocol'] == 'stim30s06V']
shortlist2 = exceldata[exceldata['Stim Protocol'] == 'stimDur300at15']
for row, dseries in shortlist1.iterrows():
    target = shortlist2[shortlist2['Sample Name'] == dseries['Sample Name']] 
    if len(target) >0:
        shortlist1['timestamp'].loc[row] = np.concatenate((dseries['timestamp'],  target['timestamp'].values[0] + dseries['timestamp'][-1]), axis=0)
        shortlist1['voltage'].loc[row] = np.concatenate((dseries['voltage'],  target['voltage'].values[0]), axis=0)
        #no add the signals together
        combined_intensity = {}
        for ckey in dseries['intensity_data']['intensity']:
            target_intensity = target['intensity_data'].iloc[0]
            combined_intensity[ckey] = np.concatenate((dseries['intensity_data']['intensity'][ckey], target_intensity['intensity'][ckey]))
        shortlist1['intensity_data'].loc[row]['intensity'] = combined_intensity
    else: # select only those samples that have both stimulation periods
        shortlist1.drop([row], inplace=True)
exceldata=shortlist1
/home/daniel/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  import sys
/home/daniel/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  
In [11]:
#pull out roi data and add each roi as separate row
exceldata = cc2.pullIntensityData(exceldata)
In [12]:
##average the response between L and R rois to produce a single entry
newframe = pd.DataFrame(columns=exceldata.columns)
short_roi_name = ['g5', 'g4', 'g3', 'a1', 'b1', 'b2', 'bp1', "b'p2a", "b'p2mp", 'Background']
for name, dseries in exceldata.groupby(['Sample Name']):
    for cshort in short_roi_name:
        same_roi = dseries[dseries['roi'].str.contains(cshort)]
        if same_roi.shape[0] >0:
            new_row = same_roi.iloc[0]
            new_row['roi'] = cshort
            new_row['intensity'] = same_roi['intensity'].mean()
            new_row['mask_index'] = same_roi['mask_index'].values
            newframe = newframe.append(new_row, ignore_index = True)
print('ROIs included in analysis:')
print(newframe['roi'].unique())
exceldata = newframe
/home/daniel/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:9: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  if __name__ == '__main__':
/home/daniel/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  # Remove the CWD from sys.path while we load stuff.
/home/daniel/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:11: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  # This is added back by InteractiveShellApp.init_path()
ROIs included in analysis:
['g5' 'g4' 'g3' 'a1' 'b1' 'b2' "b'p2a" "b'p2mp" 'Background']
In [13]:
#check parameters within data
parameters_to_compare = ['Cross']
print('These are the different conditions tested.')
for cp in parameters_to_compare:
    print('Parameter:', cp)
    print(exceldata[cp].unique())
These are the different conditions tested.
Parameter: Cross
['A83-41']
In [14]:
#Get counts
print('Number of animals tested.')
groups = exceldata.groupby(['Cross', 'roi'])
groups['roi'].count()
Number of animals tested.
Out[14]:
Cross   roi       
A83-41  Background    3
        a1            3
        b'p2a         3
        b'p2mp        3
        b1            3
        b2            3
        g3            3
        g4            3
        g5            3
Name: roi, dtype: int64
In [15]:
from bokeh.io import output_notebook, show
from bokeh.plotting import figure, output_file, show, ColumnDataSource
from bokeh.models import Range1d, Span
In [16]:
output_notebook()
Loading BokehJS ...
In [17]:
#create a dictionary holding data
raw_data={}
notroi = 'Background' #will look at all rois except background
for cgroup, frame1 in exceldata.groupby('Genotype'):
    crois = frame1['roi'].unique()
    crois = [ccroi for ccroi in crois if ccroi != notroi]
    for croi in crois:
        groupname = 'ROI: ' + croi + ' ; ' + 'Genotype: '+  cgroup 
        frame2 = frame1[frame1['roi'] == croi]
        if len(frame2) != 0:
            frame3 = cc2.intensityDataFrame(frame2)
            raw_data[groupname] = frame3.subtractBackground(exceldata)
In [18]:
volt = cc2.intensityDataFrame(exceldata).getVoltage()
timevolt = np.arange(0, len(volt))/100
plot=figure(y_range = (timevolt[0], timevolt[-1]), x_range = (0, 0.9), plot_width=600, plot_height=200)
plot=figure( plot_width=600, plot_height=200)
source = ColumnDataSource(data=dict(y=volt, x= timevolt ))
plot.line('x', 'y', source = source, line_width = 3, line_color ='red')
plot.x_range = Range1d(0, np.max(timevolt)*1.05)
plot.y_range = Range1d(0,np.max(volt)*1.05)
plot.xaxis.axis_label = 'Time (S)'
plot.yaxis.axis_label = 'Volt (V)'
#plot.line(timevolt,volt)
#output_file("StimProtocol.html", title = 'Stimulation Protocol')
show(plot)
In [19]:
from bokeh.models import LinearAxis, Range1d
from bokeh.palettes import Spectral5
from bokeh.models import Legend
In [20]:
source = ColumnDataSource(exceldata)
In [21]:
#Intensity Data (subtract background plot)
#Each figure will plot a different gentoype
#black line is mean value
#light grey lines are individual data
timeStamp = cc2.intensityDataFrame(exceldata).getTimeStamp()
for ckey in list(raw_data.keys()):
    #get the min and max range for the data
    data = np.vstack(raw_data[ckey]['intensity'])
    min1 = np.min(data)
    max1 = np.max(data)
    p = figure(plot_width=1000, plot_height=400, y_range=Range1d(min1, max1), x_range = Range1d(0, np.max(timeStamp)), x_axis_label = 'Time (S)', y_axis_label = 'Intensity - Background')
    #plot showing voltage
    p.title.text = ckey
    p.extra_y_ranges = {"foo": Range1d(start=-0, end=2)}
    p.add_layout(LinearAxis(y_range_name='foo', axis_label = 'Volt (V)'), 'right')
    c = p.line(timevolt, volt, y_range_name = 'foo', line_color = (255,0, 0), line_alpha=0.5, line_width =2)

    #plot showing mean data results
    legend_it = [('Voltage', [c])]
    c = p.line(x=timeStamp, y=np.mean(data, axis=0), line_width=2, line_color = (0, 0, 0))
    legend_it.append(('Mean', [c]))
    for row in range(0,data.shape[0]):
        c = p.line(x=timeStamp, y=data[row, :], line_width=2, line_color = (150, 150, 150), line_alpha = 0.5) #line_color = colors[cgroup])
        legend_it.append((raw_data[ckey]['Sample Name'].iloc[row], [c]))


    #p.sizing_mode = 'scale_width'

    legend = Legend(items = legend_it, location=(0, 100))
    legend.click_policy="hide"
    p.add_layout(legend, 'right')


    

    #p.x_range = Range1d(0, 450)
    p.y_range = Range1d(min1, max1)
    #output_file("Mean-RawIntensity.html", title = 'Stimulation Protocol')
    #p.patch(timevolt2, volt2, y_range_name = 'foo')
    show(p)
In [22]:
def generateGroupNames(list1):
    #list1 = list that is converted to string
    for i, l in enumerate(list1):
        if not isinstance(l, str):
            list1[i] = str(l)
    groupname = list1[0]
    for cg in list1[1:]:
        groupname = groupname + ' ' + cg
    return groupname
In [23]:
#get imaging_speed 
time = np.hstack(exceldata['timestamp'].values)
time = np.mean(time,axis=1)
seconds_per_image = np.mean(np.diff(time))
print(seconds_per_image)
1.004
In [24]:
# getting the DeltaF/F responses for each time point then placing in a dictionary
print('Getting the mean deltaF/F over for the first 8 stim periods')
deltaFF_data={}
parameters_to_compare = ['roi']
stimPeriods, stimoffPeriods = cc2.voltage(cc2.intensityDataFrame(exceldata).getVoltage()).getVoltageStopStarts()
stimPeriods = stimPeriods[:8]
stimoffPeriods = stimoffPeriods[:8]
prestim = 5 # time in s to include before stimulation
poststim = 20 # time after first stimulation to include

rois1 = exceldata['roi'].unique()
rois1 = rois1.tolist()
rois1 = [croi for croi in rois1 if not croi =='Background']
#create a pandas dataframe to hold deltaFF data
time_stamp = np.arange(0, prestim+poststim+1) * seconds_per_image
deltaFF = pd.DataFrame(columns = rois1, index = time_stamp)
#add intensity data to dictionary
for croi in rois1:
    frame = exceldata[exceldata['roi'] == croi]
    #reorganize intensity data lining the data up so the stim is synchronized
    deltaFF_sync = np.zeros([frame.shape[0], len(stimPeriods), prestim+poststim+1])
    for cstimindex, cstim in enumerate(stimPeriods):
        prestart = cstim - prestim
        stop = stimoffPeriods[cstimindex] + poststim
        #for row, cstim in enumerate(stimPeriods):
        #    intensity_sync[row, :] = intensity_sync[ cstim-prestim: cstim + poststim]
        #frame = frame[frame['roi'].str.contains('gamma')]
        dFF = cc2.intensityDataFrame(frame).getdFF(prestart, cstim, stop, exceldata, 'pos')
        deltaFF_sync[:, cstimindex, :] =dFF.deltaFF()
    deltaFF[croi] = np.mean(np.mean(deltaFF_sync, axis=0),axis=0)
    
deltaFF=deltaFF.transpose()
Getting the mean deltaF/F over for the first 8 stim periods
In [30]:
#graph mean values from all rois into one plot
p = figure(plot_width=800, plot_height=400, y_range=(0, np.max(deltaFF[:].values)), x_range = (0, deltaFF.columns.max()), x_axis_label = 'Time (S)', y_axis_label = 'deltaF/F')

p.title.text = 'Comparing reponses between rois'
legend_it = []

for row, dseries in deltaFF.iterrows():
    color = np.array(rois[row])*255
    c = p.line(x = dseries.index,  y = dseries.values, line_color  = (int(color[0]), int(color[1]), int(color[2]) ), line_width = 2)
    legend_it.append((row, [c]))
stim_time = Span(location = prestim, dimension='height', line_color = (0, 40,0), line_dash='dashed', line_width=2, line_alpha=0.5)
p.add_layout(stim_time)
legend = Legend(items = legend_it, location=(0, 0))
legend.click_policy="hide"
p.add_layout(legend, 'right')

show(p)
In [29]:
 
In [ ]:
 
In [27]:
!jupyter-nbconvert A83_Data4_FirstPass_v2.ipynb
[NbConvertApp] Converting notebook A83_Data4_FirstPass_v2.ipynb to html
[NbConvertApp] Writing 9052624 bytes to A83_Data4_FirstPass_v2.html